const _ = require('lodash/fp')
const fs = require('fs')

let idfs = fs.readFileSync('./idf.txt', { encoding: 'utf8' }).split('\n')
let idf = {}
for (let line of idfs) {
    line = line.split(' ')
    if (line.length === 2) {
        idf[line[0]] = line[1]
    }
}

const max_idf = 14

module.exports = (words, limit = 20) => {
    let count = _(words)
        .filter(word => word && word.length > 1 && isNaN(word) && !(/\d+[年月日]/.test(word)))
        .countBy()
        .value()

    return _(count)
        .keys()
        .map(word => ({ word: word, weight: (count[word] * (word in idf ? parseFloat(idf[word]) : max_idf)) }))
        .orderBy('weight', 'desc')
        .take(limit)
        .value()
}